#!/usr/bin/env python
# _*_ coding: utf-8 _*_
# @Time : 2020/12/11 22:45 
# @Author : wulinlin 
# @Version：V 0.1
# @File : crawl_list_by_redis_queue.py
# @desc : 从redis队列中读取关键字 抓取其列表页

import requests


class BaseSpider(object):
    def __init__(self):
        pass

    def do_get(self):
        pass

    def xpath(self):
        pass


class QccSpiderList(object):
    def __init__(self):
        self.user_agent = None
        self.cookie = None
        pass

    def get_html_doc(self, link, **headers):
        response = requests.get(url=link, headers=headers)
        if response.status_code == 200:
            return response.text
        return None

# 1. 从redis中获取cookie

# 2. 从redis中获取对应的搜索关键字

# 3. 抓取列表页

# 4. 解析数据入库

# 5. 如果没有抓取成功 将该关键字再次插入redis队列中